{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gradient_check import eval_numerical_gradient_array\n",
    "import numpy as np\n",
    "from layers import *\n",
    "import matplotlib.pyplot as plt\n",
    "import itertools\n",
    "np.random.seed(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1.1 gradient check: affine transformation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.19169953  0.77598634  1.0997028   1.80174338]\n",
      " [ 2.26343588 -2.09109747 -3.81138775  0.81421387]\n",
      " [ 0.14705629 -0.13054605 -0.24073457  0.06949683]]\n",
      "[[-0.19169953  0.77598634  1.0997028   1.80174338]\n",
      " [ 2.26343588 -2.09109747 -3.81138775  0.81421387]\n",
      " [ 0.14705629 -0.13054605 -0.24073457  0.06949683]]\n"
     ]
    }
   ],
   "source": [
    "# gradient checking: compare the analytical gradient with the numerical gradient\n",
    "N = 2\n",
    "D = 3\n",
    "M = 4\n",
    "x = np.random.normal(size=(N, D))\n",
    "w = np.random.normal(size=(D, M))\n",
    "b = np.random.normal(size=(M, ))\n",
    "dout = np.random.normal(size=(N, M))\n",
    "\n",
    "out, cache = affine_forward(x, w, b)\n",
    "grad = affine_backward(dout, cache)[1]\n",
    "ngrad = eval_numerical_gradient_array(lambda w: affine_forward(x, w, b)[0], w, dout)\n",
    "print(grad)\n",
    "print(ngrad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1.2 gradient check: ReLU"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[ 1.70194181  1.06535754 -0.00408705]\n",
      " [-0.22360542 -0.         -2.30814003]]\n",
      "[[ 1.70194181  1.06535754 -0.00408705]\n",
      " [-0.22360542  0.         -2.30814003]]\n"
     ]
    }
   ],
   "source": [
    "N = 2\n",
    "D = 3\n",
    "M = 4\n",
    "x = np.random.normal(size=(N, D))\n",
    "w = np.random.normal(size=(D, M))\n",
    "b = np.random.normal(size=(M, ))\n",
    "dout = np.random.normal(size=(N, D))\n",
    "\n",
    "out, cache = relu_forward(x)\n",
    "grad = relu_backward(dout, cache)\n",
    "ngrad = eval_numerical_gradient_array(lambda x: relu_forward(x)[0], x, dout)\n",
    "print(grad)\n",
    "print(ngrad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.1.3 gradient check: Softmax Loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[[-0.08002098  0.01998052  0.02000876  0.02002523  0.02000647]\n",
      " [ 0.02002995  0.02001225  0.0200058   0.01997098 -0.08001898]\n",
      " [ 0.02000956  0.020007    0.01999959  0.02000844 -0.08002459]\n",
      " [ 0.01995912  0.01999004  0.0200132   0.02001691 -0.07997926]\n",
      " [ 0.02000124  0.02002489 -0.08002547  0.01999769  0.02000165]\n",
      " [ 0.01998262 -0.07998815  0.02001365  0.01995633  0.02003555]\n",
      " [ 0.02000771  0.02003053  0.01997868 -0.08003298  0.02001606]\n",
      " [-0.08001727  0.02003667  0.02000045  0.01999148  0.01998867]\n",
      " [ 0.01998429  0.02001077 -0.08001233  0.0200117   0.02000557]\n",
      " [ 0.02000473  0.0199983   0.01998187 -0.08000583  0.02002093]]\n",
      "[[-0.08002098  0.01998052  0.02000876  0.02002523  0.02000647]\n",
      " [ 0.02002995  0.02001225  0.0200058   0.01997098 -0.08001898]\n",
      " [ 0.02000956  0.020007    0.01999959  0.02000844 -0.08002459]\n",
      " [ 0.01995912  0.01999004  0.0200132   0.02001691 -0.07997926]\n",
      " [ 0.02000124  0.02002489 -0.08002547  0.01999769  0.02000165]\n",
      " [ 0.01998262 -0.07998815  0.02001365  0.01995633  0.02003555]\n",
      " [ 0.02000771  0.02003053  0.01997868 -0.08003298  0.02001606]\n",
      " [-0.08001727  0.02003667  0.02000045  0.01999148  0.01998867]\n",
      " [ 0.01998429  0.02001077 -0.08001233  0.0200117   0.02000557]\n",
      " [ 0.02000473  0.0199983   0.01998187 -0.08000583  0.02002093]]\n"
     ]
    }
   ],
   "source": [
    "N = 2\n",
    "D = 3\n",
    "M = 4\n",
    "x = np.random.normal(size=(N, D))\n",
    "w = np.random.normal(size=(D, M))\n",
    "b = np.random.normal(size=(M, ))\n",
    "dout = np.random.normal(size=(N, D))\n",
    "\n",
    "num_classes, num_inputs = 5, 10\n",
    "x = 0.001 * np.random.normal(size=(num_inputs, num_classes))\n",
    "y = np.random.randint(num_classes, size=num_inputs)\n",
    "\n",
    "ngrad = eval_numerical_gradient_array(lambda x: softmax_loss(x, y)[0], x, 1)\n",
    "loss, grad = softmax_loss(x, y)\n",
    "print(grad)\n",
    "print(ngrad)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [],
   "source": [
    "# TODO: put the path to your 'hw6_mds189', which should contain a 'trainval' and 'test' directory\n",
    "path = '/Users/dhanush/Documents/GitHub/cs189/hw6/hw6_mds189/trainval/'\n",
    "\n",
    "from data_utils import load_mds189\n",
    "# load the dataset\n",
    "debug = False  # OPTIONAL: you can change this to True for debugging *only*. Your reported results must be with debug = False\n",
    "feat_train, label_train, feat_val, label_val = load_mds189(path,debug)\n",
    "from solver import Solver\n",
    "from classifiers.fc_net import FullyConnectedNet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished training model # 8/648"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/dhanush/Documents/GitHub/cs189/hw6/resources/problem2/layers.py:137: RuntimeWarning: divide by zero encountered in log\n",
      "  loss = -np.sum(np.log(probs[np.arange(N), y])) / N\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished training model # 648/648"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "data = {\n",
    "      'X_train': feat_train,\n",
    "      'y_train': label_train,\n",
    "      'X_val': feat_val,\n",
    "      'y_val': label_val}\n",
    "\n",
    "best_val = 0.0\n",
    "best_model = None\n",
    "best_combo = None\n",
    "\n",
    "def train_model(combo):\n",
    "    hd,lrd,ne,bs,lre,ws = combo\n",
    "    model = FullyConnectedNet(input_dim=75,\n",
    "                              hidden_dims=hd,\n",
    "                              weight_scale=ws)\n",
    "    solver = Solver(model, data,\n",
    "                    update_rule='sgd',\n",
    "                    optim_config={\n",
    "                      'learning_rate': lre,\n",
    "                    },\n",
    "                    lr_decay=lrd,\n",
    "                    num_epochs=ne, \n",
    "                    batch_size=bs,\n",
    "                    verbose=False\n",
    "                    #print_every=100\n",
    "                   )\n",
    "    solver.train()\n",
    "    return solver\n",
    "    \n",
    "lr_decay = [0.95,0.99,1.0]\n",
    "num_epochs = [10,15]\n",
    "batch_size = [64,128,256]\n",
    "learning_rate = [1e-4,1e-3,1e-2]\n",
    "weight_scale = [1e-4,1e-3,1e-2]\n",
    "hidden_dims = [[100]*i for i in range(1,5)]\n",
    "\n",
    "prog = 1\n",
    "combos = [combo for combo in itertools.product\\\n",
    "(hidden_dims,lr_decay,num_epochs,batch_size,learning_rate,weight_scale)]\n",
    "total = str(len(combos))\n",
    "\n",
    "for combo in combos:\n",
    "    model = train_model(combo)\n",
    "    avg_val = np.mean(model.val_acc_history)\n",
    "    if avg_val > best_val:\n",
    "        best_val = avg_val\n",
    "        best_model = model\n",
    "        best_combo = combo\n",
    "    print(\"\\rfinished training model # \"+str(prog)+\"/\"+total,end=\"\")\n",
    "    prog += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XHW9//HXJ5M9aZK26Zp0hbZ0pS0pglWkFgVECnorVkSF3wX8oYh4FS3qjyL3h7c/8adc709QQHFDsAICaqFC2bkgbVnSje5bGtqkS9I2zTrz/f1xJtMknTSTZXImmffz8cjjzDlzzsln0s75nPNdzTmHiIgIQIrfAYiISOJQUhARkQglBRERiVBSEBGRCCUFERGJUFIQEZEIJQUREYlQUhARkQglBRERiUj1O4DOKiwsdGPHjvU7DBGRPmXNmjUHnHNDOtqvzyWFsWPHsnr1ar/DEBHpU8xsVyz7qfhIREQilBRERCRCSUFERCKUFEREJEJJQUREIvpc6yMRkZ70xNt7uWvFJsqrahlZkMUtF07i8llFfofVSm/GqKQgIknribf3cuvja6ltDAKwt6qWWx9fC9Dli25PX8DjEeOpKCmISNK6a8WmyMW2WW1jkB8u38jZ4waRmRYgIzWFzLQAgRTr8HydvYA756hvCnG8IUhNfZO3bGjieH142dDE7U9tiBrjXSs2KSmIiHRHUzDEtsoa1u6tZm1ZFXuraqPuV3G0ng8ufb7VtrSAkZkaICMtQGaalygy01LITA1EXr+29QC1jaFWx9U2BvnOY6U8/ObuKBf9IMGQ69JnKW8n9u5SUhCRiL5Qvh6rpmCIrZXHWFtWzbq91azdW82G949QF75oZ6cHSA+k0BAMnXTswOw0Fl98BnWNIWobg9Q1BqlrDFHXGKS+6cTr5u01DU0crAmdlBCa1TeFcEBhbjqjM7LJSQ+QnZ5KbkYq2RkBctJTyU4PkJPRZpmeyqL73mDfkbqTzjmyIKtH/17NlBREBIhe9LH48VKCoRD/ctaoLp+zp5NMtHN+csaIUyaAnPQAU0fmc+XZY5henMf0onzGFeby13fLW31mgKy0AEsundqlOOcufT7q00dRQRbLvnxulz7v4ovPiBrjLRdO6tL5OmLOde3RxS8lJSVOYx+J9JzKo/Ws2XWIb/75XWrqg1H3SU9Nidzd5mS0WaYHyM5IPen9jfuq+fOqMhqCJ64xGakp3HD+aXxkYofjskX10uZK7n1xG/VNJ+7IzSBg0LypOQFMK8oPJ4ACxhXmtFsn0JOJq21iBe8C/h+fnt7tyubuxmhma5xzJR3up6Qgkjycc+w8eJxVOw+xaschVu86zI4DNR0e9z8/chrHG5qoqQ96y4Ygx+ubOBauHG1+r22FaG/JyQhw5+XTmVaUz/jCHFJiqBSOl1VP/ZJRb93FUFdJhQ1hz+xbmLPgy907aekyWHkHVJdBfjHMvw1mXNGpU8SaFFR8JDHpT2XN/UlH/y5NwRDry4+wauchVu88zOpdhzhwrAGAguw0SsYMYtGcUcwZN4gbH3qL8uqTy66LCrJYfPEZMcUTDDlqG72E8YEfrqS9W84Hr5nT6c8KcM2Dq6JuP14fTIz/j6XLmLN2CVALBsOpZPjaJTB2YKcv4i3PyV9vgsZwsVT1Hm8dun7OU1BSkA71djtpiU17dQCb9h0lPTWF1bsO8fbuKo43eO+PGpTFeROGMGfcIOaMHcj4wtxWd9Tfvqj7ZdeBFCM3w6tAHVmQxVlHnuXbqcsYaQcod4X8qOkK1uR9jHmThnbpMxed4pxd1t278GAT1B/xfv7x/RMX72aNtbD8W1C1G5rqIVjvLSM/dRBs8JbN21ruU7UbXPDkc668Q0lB/NFeW+7bn1pPVnqAwTnpDMpJZ3BOBnlZqZjF9uie6E8fvVVJ2tlzHm9o4uCxBu78+8aT/l3qGkPc+9I2Ugwmj8jjipJRlIwdSMmYQQzPzzzleZvj6KnPfPeULUxb8wBZ5j2ZFNsB/k/aA6ybMhb4qL/ndA6CjfDOH+GZ73gXZPDuwp/8Kuz6bxg6GeqOQH11eHkk+rKx4+I36qrh+X8HDFIzvJ9ABqRmQmq6twyEl+nZkDroxPrhHdHPWV0W++ftBCUFadf71bX8Y/3+dttyV9U28uXfr2m1LTXFGNScJHLTGZSTEUkaXuLwlu/sOcxPn9sSaR2SaD1J49XT9eRzllLX2MS5pxVysKaBQ8caOFhTH3l9qKbBe13T/Lo+8jcDWJDy6kl3zU+FPsQ7Sz5OXmZap2O8PPAal2fcAZllkFEMgduAGO5GQyE4fhCO7YOj++HYPuasvxPCF+9mWdbAnNLboOYlCKRCShoE0iAlNbyMtn5ivzkblkY/5/r/Dem7oPG4dxcdWda2v63t3XezYAOsefDEemomZORBZj5k5nmv80ae2JaRd2L7s//L+zu0lVcEN73jfaYYb5oidr/uJau28os7d54YKSlIK9sqj7Fi/T5WrNvHu2XVgHehb4rSwWZYXga/+tIcDhyrb3HRar6wNXCopp61h6s4WNPA0bqmDn93cyeflzZXkp+VRkF2GgVZaRRkp5Pf4nVBVhp5WWmtWpN09yLeGAy1aov+w+Ub+VjwJb6d3vqC+4O/emNINgRDNAZDNDaFaAy6E+vB8HpTm/VgiJUb9lPX1LZjU4jFj6+LGlNmWgqDczIYnOsl2AnDcsNJ1Uu07yy/j++HHiC7xV3z0rQHGJSWTl7mJR1+5pO0V3ZdexiKSyIX+6jLmgoIdfxvDHh35VW7vDv1UKNX/BJqjL7ebq1EG/VH4K3fQVpWm59syB504nWrZRY8/7/bOaHBLVu9C31qemwxgHfRb/k3BO/3XHB7587T0vzbop9z/m1dO18H1PooyTnnWLu32ksE6/ezteIYAGcW53PhtOFcOHU4a8uqu93MrqEpxOHjDZEE8oVfvdnuvqMGZVF1vLHDRJKXmeoliew0Nu072qqZ4ok4U/jgaYXURelw1LITUtuktyDlVZamnbjgAhx36SxuvJanQh9qN6ZAipEWMNICKWSkppAWaP4xtlW2X8zw48+c2fqJKjed7PQo92zOeRfU99+l8fGvkNZ07ORdMCwzDywFLOAtUwIt1q3Nevj9yk3exTgW2YUwYDjkDmt/+ZtPwpEoRRz5o+Ab0RPhSULBFsmiEX4xF46Ud++cLf10Wjt34V08H/RIS6F4nFOtj6RdTcEQq3YeZsX6fTy7wSseCqQYHxg3iC+cM4aPTx3GiPwTvSVPG5ILdK+sOT01hWF5mQzL88q1iwqy2u3k88q3PxqJ80hdE1XHG6iqbaT6eCOHjzdQdbwxvO5trzreGDUhgHcnvu9IXWQYgoKsNG88m+ZhClK97dkBx7DGPQyr20ZhzTbGb/stmbQupsi2Bn6U/gB3zHKEBk2AIROxIZNIzRkYufifanycU3VsWnhWlKKAUAgObYf334H33z3xU1cFQHuFQ4aDGYvAhbwiEhfyLq7OtVlv+X4I9p/iIrjoYRgwDHKHQ+5Q7464Ixcs6f4dbkrA+yFcH3LBD3r2rjked+Ezruj5CuB4nLMdelLop9qWr998wQQG5aTzzLp9PLdxP4ePN5KRmsKHJwzhwqnDuGDyMAbmdPHxtovxvfqXe7iZRyLFM3eziA996itd7knaXquU1xa3qIB0zrvTrNgA+9eHlxvgwCavLBnAAjgXJNrl3QEWSD+xL0DOECicCIUTwsvw6/zRkHJiypJTfuYZw+DgFu+iXx5OAvtKoSH8JBBIh2FTYcSZJ37+9MXu34m3lER3zXGPMQGp81oSi9arstmAjFQ+OnkoF00dznkTh5CTEePDYhy+iE1Pfo3U4Il28U2BTFIv+68unXfVU79k2prvR1qlANS6dHaecS2TJ0zwEsD+DV4SCN9pAzBgpHexHTYFhoaXhRPhv85q/wL59Xfh8E44sAUObA7/bPESS+3hE/umZsLgCSeSxfEDBNf8nkCoPrJLyAKkFIyGo/ugKXy3mpoFw6e3TgBDzji5TLptHQB4d7mX/qxr/zY9fT5JKEoKSay9YorBOem8fut80lM7OeFeVy4WoSDUHw0324vSpG/lv3tN/dpKy4HpCzsXH8DaR0/dNDB9QPjCP8VLAkOneOtZA6Pv39ULZM3BFoli84nEUbXLK6aJJpABc649kQAKJ4SLTGKgu2aJkZJCkqo8Ws+cO5+L+p4BO5Z2oVVKe8UK6Tkw8aLobbcbjnb+9zTLHd75Y47ta+cNg5tLvTv8zjYF7MkLZGMd3Dmc6K1pDG6virJdpOeoojnJOOf485oy7vz7xnb36fJQu+11kmmo8cq/m9toFw6FjBZtuaMuw+/fN6+XysOLoWB0588HPVu5l5bpxdKL7c1FukJJoR/YcaCG7z6+lte3H+TssYOYP3kodz+3pXtD7YZCsGUFvHEP7bYVzx8FN73VtaB7omVKS73clrtL+kKMkvSUFPqwhqYQ97+ynf9cuYWM1BT+49PT+WzJKFJSjGF5mV1rQlp/DN59GN64Fw5t83piTv00bHr6REUo9EyzPei54pmePl889IUYJempTqGPemv3YW59bC2b9h/lkukjWHLpFIbmnXpsm1OqLoM374M1v/EqhotK4NyvwOQFXpt0VUCK9GmqU+injtU3cdcz7/G7N3YxPC+TB75YwgVThnX9hGWr4fWfw4YnAeclgXO/CqPObr1fL3aeERH/KCn0Ic9u2M9tT65j35E6vnTuWL514SRyY+1n0FKwCd77K7x+D5S96VUCn3MDfODLXa+UFZF+QUmhD6g4UseSp9bz9Lp9nDF8APd8fjazRrfTvr5ZtOKeCR/3Bg178z6vFczAcXDxj2DmlZAxoHc+jIgkNCWFBBYKOR5etZulT79HfVOIWy6cxPXnjSct0EHns2ijXT5xA5ACoQYY8yG4+P94fQxi7SQlIklBSSFBtB2r6Ivnjua5jRWs2nmYc8cP5oefns64wpzYTrbyjpNnfwo1eUMGX/ssjJzZ8x9ARPoFJYUEEG0ugP94ehNZaSn8aOEMPnNWcWyzmdUehs3/iN5BCrxEoYQgIqegpJAAok13CZCfnc4VJaNOfXD1Xti0HN77G+x81XsisJTo4+yo56yIdEBJIQGUtzPd5f7qupM3OgeV73lJ4L2/Q/nb3vbCifDBr8EZn4SD2+BvX1fPWRHpNCWFBDCynQlnImMVhUJQtupEIji0zdteVALzl3iJYMjEEwcWl3iDv6mzmYh0kno0J4DH1+zhpcfu4ZYWE8T8F5/hsg+eyQcbXveGmKip8CYvH3cenHEJTPoE5I3wO3QR6SPUo7kPGfv+ci5KazMBO/dibwDpuTDhY97TwOkXQFaBv8GKSL+mpOAz5xzFb/241QTx4M19QHYh/NsGSM3wJTYRST6dnIKrc8zsIjPbZGZbzWxxlPdHm9kLZva2mZWa2SfiGU8iWr3rMIXBiuhvHj+ohCAivSpuScHMAsDPgYuBKcDnzGxKm92+Dyxzzs0CFgH3xCueRHXfy9vZZ4XR31QTUhHpZfF8Ujgb2Oqc2+6cawAeAS5rs48D8sKv84HyOMaTcHYcqOG5jfvZWXTpyW+qCamI+CCeSaEIaNm1tiy8raXbgavMrAxYDnwtjvEknF+9up20lBTOytjrTVifVwyYN6NZRxPEi4jEQTwrmqONy9C2/evngN845/6vmZ0L/N7MpjnXujuumV0PXA8wenT/GNr5UE0Dj64p47opQTK2/AM+8h2Y912/wxKRJBfPJ4UyoOUYDcWcXDz0r8AyAOfc60AmcFIBu3PuPudciXOuZMiQIXEKt3f94Y1d1DWGuD5tOQQyYM51fockIhLXpLAKmGBm48wsHa8i+ak2++wG5gOY2WS8pFAZx5gSQl1jkN+9vpNLT08lf9OjMPNzkNs/kp2I9G1xSwrOuSbgRmAFsBGvldF6M7vDzBaEd/smcJ2ZvQs8DFzt+loX6y548p29HDjWwC0DX4VgPZzzVb9DEhEB4tx5zTm3HK8CueW221q83gDMjWcMiSYUctz/yg5mDU9n1LaHYOLFrcctEhHxUVw7r8nJXtpcydaKY9w2uhQ7ftAb2VREJEEoKfSy+1/ZzogB6cwsewhGzoIxH/Q7JBGRCCWFXrRubzX/ve0gSybtxg5t854SYplRTUSklygp9KIHXtlOTnqAC6qWQf5omNy2g7eIiL+UFHrJ+9W1/K30ff5tylFSy96Ac26AgAapFZHEoqTQS37z2k5CzvG5pichIx9mf8HvkERETqKk0AuO1jXyx3/u5qozIHvb36HkGsgY4HdYIiInUVLoBX9atYej9U18LetZsBT4wJf9DklEJColhThrCoZ48LWdzBudxpAty2D6ZyBvpN9hiYhEpaQQZ0+v28feqlq+O/R1aKyBc2/0OyQRkXYpKcSRc477X9nOxMHpnL7zjzB+Hgyf5ndYIiLtUlKIozd3HKK0rJrbx23Eju3TkBYikvCUFOLo/ld2MDArlXP2/xGGToXTPup3SCIip6SkECfbKo+x8r39fP+M90mp3AgfvFFDWohIwlNSiJNfvbqDtEAKl9Y8BgNGwLSFfockItIhJYU4OHisnsfWlPHVM46Tvvtlr19CarrfYYmIdEhJIQ7+8MZu6ptCXJPyd0jLgbOu9jskEZGYKCn0sOb5lz99GuRtfRJmfxGyBvodlohITJQUethf3t7LwZoGvpn/AriQNxqqiEgfoaTQg0IhxwOvbKdkRCojt/0JplwGA8f4HZaISMyUFHrQi5sr2FZZw5KiNVj9EXVWE5E+R0mhB9338naK89KYtuchGDMXis7yOyQRkU5RUugh6/ZW88b2Q/xgwlasukwD34lIn6Sk0EPuf2U7uRkBzj/wMAw+HSZe5HdIIiKdpqTQA8qrvPmXvz3pAIH9pd5TQor+tCLS9+jK1QMefG0HAJ9pfAKyC+HMRT5HJCLSNUoK3XS0rpFH3tzDNRMbyNrxLJx9HaRl+R2WiEiXKCl0U/P8yzdkPA2pmTDnWr9DEhHpslS/A+irnnh7Lz9a8R7lVXWMCByhYMvjMOvzkFPod2giIl2mJ4UueOLtvdz6+FrKq+oAWGQrsGAjz+VreGwR6duUFLrgrhWbqG0MApBJPV8IPMtzodks+e8GnyMTEekeJYUuKK+qjbxeGHiZQXaM+5suabVdRKQvUlLogpEFXuuiFEL8a2A574ROY5WbFNkuItJXKSl0wS0XTuJTqa+xKuMGxqXsZ5RVsDD9dW65cJLfoYmIdItaH3XB5YHXuCj1ATKpB2CwHWVp6gOkBs4ErvA3OBGRbtCTQlesvCOSEJqlButg5R0+BSQi0jOUFLrAVZdFf6O97SIifYSSQhc05Y6M/kZ+ce8GIiLSw2JKCmb2mJldYmZKIsCGKd8g6Kz1xrQsmH+bPwGJiPSQWC/y9wJXAlvMbKmZnRHLQWZ2kZltMrOtZra4nX2uMLMNZrbezP4YYzy+ejVrHoddLi41CzDIHwWX/gxmqJJZRPq2mFofOeeeA54zs3zgc8CzZrYHuB/4g3Ouse0xZhYAfg58DCgDVpnZU865DS32mQDcCsx1zh02s6Hd/kS9YM/7FRSmHIUPfx8+covf4YiI9JiYi4PMbDBwNXAt8Dbwn8Bs4Nl2Djkb2Oqc2+6cawAeAS5rs891wM+dc4cBnHMVnYreJ437wnlt6GR/AxER6WGx1ik8DrwCZAOXOucWOOf+5Jz7GpDbzmFFwJ4W62XhbS1NBCaa2Wtm9oaZJfwclqGQI7tqk7cybIq/wYiI9LBYO6/9P+fc89HecM6VtHOMRdnmovz+CcD5QDHwiplNc85VtTqR2fXA9QCjR4+OMeT4KK+uZVxoN02pmaQWjPU1FhGRnhZr8dFkMytoXjGzgWb2lQ6OKQNGtVgvBsqj7POkc67RObcD2ISXJFpxzt3nnCtxzpUMGTIkxpDjY2vFMSbaHuoGTtQ8zCLS78R6Vbuu5d17uA7gug6OWQVMMLNxZpYOLAKearPPE8A8ADMrxCtO2h5jTL7YWnGMSSl7SBsx1e9QRER6XKxJIcXMIsVB4ZZF6ac6wDnXBNwIrAA2Asucc+vN7A4zWxDebQVw0Mw2AC8AtzjnDnb2Q/Sm98v3MMSOkDFymt+hiIj0uFjrFFYAy8zsF3j1Av8TeKajg5xzy4Hlbbbd1uK1A/4t/NMnNO1b771QJbOI9EOxJoXvAF8GbsCrQP4H8EC8gkpUzjmyD2/2VoYqKYhI/xNr57UQXq/me+MbTmI7cKyB0U07qcsqIDN3mN/hiIj0uJiSQrjn8X8AU4DM5u3OufFxiishbak4yqQUr+VRpkVrcSsi0rfFWtH8IN5TQhNea6HfAb+PV1CJatv+I0y0MtJUySwi/VSsSSHLObcSMOfcLufc7cBH4xdWYqos20au1ZFdNN3vUERE4iLWiua68LDZW8zsRmAv0CcGr+tJoXDLIxumPgoi0j/F+qRwM964RzcBZwFXAV+KV1CJKququeVRTCOHi4j0OR0+KYQ7ql3hnLsFOAZcE/eoElB1bSNFjTs4mj2cAZn5focjIhIXHT4pOOeCwFktezQno60Vx5hke6gfNMnvUERE4ibWOoW3gSfN7M9ATfNG59zjcYkqAW3bd4jLrZzaEQs63llEpI+KNSkMAg7SusWRA5ImKRzes5F0C5I6eobfoYiIxE2sPZqTsh6hpWB4trUUjXkkIv1YrD2aH+TkCXJwzv2PHo8oQeVUbSJICoHCiX6HIiISN7EWH/2txetM4FOcPGFOv3W8oYkR9Tuozh3NoLTMjg8QEemjYi0+eqzlupk9DDwXl4gS0PbKmnDLo9l+hyIiElddnU9yAuDvZMm9aEd5BWNSKkgfqZ7MItK/xVqncJTWdQr78OZYSApVu0oByBsz0+dIRETiK9biowHxDiSRhfZ7LY80L7OI9HcxFR+Z2afMLL/FeoGZXR6/sBJLdtVmGiwDBo71OxQRkbiKtU5hiXOuunnFOVcFLIlPSImloSnEiPrtHMoeBykBv8MREYmrWJNCtP1ibc7ap+06WMNEK6N+kEZGFZH+L9aksNrMfmJmp5nZeDP7KbAmnoElip179jDUqkjXbGsikgRiTQpfAxqAPwHLgFrgq/EKKpEc2fUuAIPGqeWRiPR/sbY+qgEWxzmWhNTc8ihDTwoikgRibX30rJkVtFgfaGYr4hdW4sip2syxlAEwYLjfoYiIxF2sxUeF4RZHADjnDpMEczQHQ47h9ds4mHMaJPccQyKSJGJNCiEziwxrYWZjiTJqan9TdqiGCZTRoNnWRCRJxNqs9HvAq2b2Unj9POD6+ISUOMp2bmaM1XJo5HS/QxER6RWxVjQ/Y2YleIngHeBJvBZI/Vp1eMyjwePV8khEkkOsA+JdC3wdKMZLCucAr9N6es5+x+1fD0BusZ4URCQ5xFqn8HVgDrDLOTcPmAVUxi2qBJFTvYWDgULIKuh4ZxGRfiDWpFDnnKsDMLMM59x7QL+ufXXOMbxuOwdzTvc7FBGRXhNrRXNZuJ/CE8CzZnaYfj4d5/7DxxjHXrYOmud3KCIivSbWiuZPhV/ebmYvAPnAM3GLKgGUbV/HcGsivUg9mUUkeXR6pFPn3Esd79X3HQ2PeVQ4bpbPkYiI9J6uztHc77mKjQQxCsZotjURSR5KCu3IrdrMvkARlpbldygiIr1GSaEdw+vV8khEko+SQhSHDh+m2O2ncbBmWxOR5BLXpGBmF5nZJjPbambtzsdgZgvNzIWH0vDd+1vfJcUcGUWqTxCR5BK3pGBmAeDnwMXAFOBzZjYlyn4DgJuAf8Yrls46sjvc8mi8Wh6JSHKJ55PC2cBW59x251wD8AhwWZT9/h34EVAXx1g6Z/8G6lwaQ0dP9jsSEZFeFc+kUATsabFeFt4WYWazgFHOub+d6kRmdr2ZrTaz1ZWV8R9yKefIFspSR5OS2uluHCIifVo8k0K0qcoiE/OYWQrwU+CbHZ3IOXefc67EOVcyZMiQHgwxuhF12zmcq5ZHIpJ84pkUyoBRLdaLaT1e0gBgGvCime3EG477Kb8rm48e3s8QDtOglkcikoTimRRWARPMbJyZpQOLgKea33TOVTvnCp1zY51zY4E3gAXOudVxjKlD+7a8DUCmxjwSkSQUt6TgnGsCbgRWABuBZc659WZ2h5ktiNfv7a5ju73Z1tTySESSUVxrUp1zy4Hlbbbd1s6+58czlli5ig1UuxyKRo33OxQRkV6nHs1tDKjezO7UMaSmBvwORUSk1ykptOQcI+p3qOWRiCQtJYUW6g7tJpfjNAxWpzURSU5KCi1UbPVaHmWp5ZGIJCklhRYiLY9Om+lzJCIi/lBSaKliI++7QYwpKup4XxGRfkhJoYUBRzazJ3UMmWlqeSQiyUlJoVmwiWH1uziklkciksSUFMKaDmwlnUaa1PJIRJKYkkLYge3vAJBZNN3nSERE/KOkEHZsTylBZwwdP8PvUEREfKOkEGYVG9nphjN+ZKHfoYiI+EZJIWzAEW/Mo9wMzbYmIslLSQGgsZbChr0a80hEkp6SAhDa/x4pOJoKNduaiCQ3JQWgate7AGQVqZJZRJKbkgJey6N6l8awseqjICLJTUkBr+XRFlfE6cML/A5FRMRXSgpA3pEt7AyMZlBOut+hiIj4Skmh9jD5TZUczp3gdyQiIr5L+qTg9m8AIKSWRyIiSgrH9ngT62QVa8wjEZGk775bs2ctzmUzcpQ6romIJP2TglVsYJMr5vRhA/wORUTEd8mdFJwj7+hWttsYhuVl+B2NiIjvkjspHCknK3iUqgGnY2Z+RyMi4rvkTgoVGwEIFqons4gIJHlSqN27FoBstTwSEQGSvPXR8T2lVLuBjCoq8jsUEZGEkNRPCla5kc2hYk4fmut3KCIiCSF5k0IoyICj29hioykemO13NCIiCSF5k8KhHaS5BqpyJxBIUcsjERFI5qRQsR6A0BC1PBIRaZa0Fc0N768n1Rk5RVP9DkUkqTU2NlJWVkZdXZ3fofQLmZmZFBcXk5aW1qXjkzYp1JaVctANY9yIQr9DEUlqZWVlDBgwgLGtflMdAAAO/ElEQVRjx6oTaTc55zh48CBlZWWMGzeuS+dI2uKjlMr32OxGqeWRiM/q6uoYPHiwEkIPMDMGDx7craeu5EwKjbXkHNvFFkYxZnCO39GIJD0lhJ7T3b9lciaFA5tJIcShnNNJT03OP4GIeKqqqrjnnns6fdwnPvEJqqqq4hCRv+J6RTSzi8xsk5ltNbPFUd7/NzPbYGalZrbSzMbEM56I8JhHoSGabU2kr3ni7b3MXfo84xb/nblLn+eJt/d263ztJYVgMHjK45YvX05BQUG3fnciiltSMLMA8HPgYmAK8Dkzm9Jmt7eBEufcDOBR4Efxiqel4L511LtU8oqUFET6kife3sutj69lb1UtDthbVcutj6/tVmJYvHgx27ZtY+bMmcyZM4d58+Zx5ZVXMn26Nyba5ZdfzllnncXUqVO57777IseNHTuWAwcOsHPnTiZPnsx1113H1KlT+fjHP05tbW13P6pv4tn66Gxgq3NuO4CZPQJcBmxo3sE590KL/d8AropjPBF1e9exyxVx2rD+l+VF+rIf/HU9G8qPtPv+27uraAiGWm2rbQzy7UdLefjN3VGPmTIyjyWXtt/0fOnSpaxbt4533nmHF198kUsuuYR169ZFWu/8+te/ZtCgQdTW1jJnzhz+5V/+hcGDB7c6x5YtW3j44Ye5//77ueKKK3jssce46qpeuZz1uHgWHxUBe1qsl4W3tedfgafjGE9ESuVG3lPLI5E+p21C6Gh7V5x99tmtmnP+7Gc/48wzz+Scc85hz549bNmy5aRjxo0bx8yZMwE466yz2LlzZ4/F09vi+aQQrQrcRd3R7CqgBPhIO+9fD1wPMHr06O5FVVtFVu0+NrvzuXiIkoJIIjnVHT3A3KXPs7fq5KKZooIs/vTlc3skhpycEy0SX3zxRZ577jlef/11srOzOf/886M298zIODFzYyAQ6NPFR/F8UigDRrVYLwbK2+5kZhcA3wMWOOfqo53IOXefc67EOVcyZMiQ7kUVrmQ+kH06WemB7p1LRHrVLRdOIiut9fc2Ky3ALRdO6vI5BwwYwNGjR6O+V11dzcCBA8nOzua9997jjTfe6PLv6Svi+aSwCphgZuOAvcAi4MqWO5jZLOCXwEXOuYo4xnJChVel4TTmkUifc/ksrwT6rhWbKK+qZWRBFrdcOCmyvSsGDx7M3LlzmTZtGllZWQwbNizy3kUXXcQvfvELZsyYwaRJkzjnnHO6/RkSXdySgnOuycxuBFYAAeDXzrn1ZnYHsNo59xRwF5AL/Dnc4WK3c25BvGICCO3fQI3LYtCIrnUBFxF/XT6rqFtJIJo//vGPUbdnZGTw9NPRqzqb6w0KCwtZt25dZPu3vvWtHo2tt8V17CPn3HJgeZttt7V4fUE8f380DeXr2ORGMWFYXm//ahGRhJdc3XmdI3BgI5tCozhNLY9ERE6SXEnh6D7SGqrZ5DQFp4hINMmVFMKVzBWZ48nP6tpY4yIi/VlSJgW1PBIRiS6pJtlxFRs44AoYPqJnWy6IiPQXSfWk0Fi+jo0hDW8hIl2Xm+tdP8rLy1m4cGHUfc4//3xWr159yvPcfffdHD9+PLKeKENxJ0dSKF0GP51KWkUps1K28IFjK/2OSES6qnQZ/HQa3F7gLUuX+RLGyJEjefTRR7t8fNukkChDcff/pFC6DP56E1SXYcAAq2PCP7/n238kEemGyPd5D+C85V9v6tb3+Tvf+U6r+RRuv/12fvCDHzB//nxmz57N9OnTefLJJ086bufOnUybNg2A2tpaFi1axIwZM/jsZz/bauyjG264gZKSEqZOncqSJUsAb5C98vJy5s2bx7x584ATQ3ED/OQnP2HatGlMmzaNu+++O/L7emOI7v5fp7DyDmhs/Yezplpv+4wrfApKRKJ6ejHsW9v++2WrINhmiLTGWnjyRljz2+jHDJ8OFy9t95SLFi3i5ptv5itf+QoAy5Yt45lnnuEb3/gGeXl5HDhwgHPOOYcFCxa0O9XlvffeS3Z2NqWlpZSWljJ79uzIe3feeSeDBg0iGAwyf/58SktLuemmm/jJT37CCy+8QGFhYatzrVmzhgcffJB//vOfOOf4wAc+wEc+8hEGDhzYK0N09/8nheqyzm0XkcTVNiF0tD0Gs2bNoqKigvLyct59910GDhzIiBEj+O53v8uMGTO44IIL2Lt3L/v372/3HC+//HLk4jxjxgxmzJgReW/ZsmXMnj2bWbNmsX79ejZs2NDeaQB49dVX+dSnPkVOTg65ubl8+tOf5pVXXgF6Z4ju/v+kkF8cftSMsl1EEssp7ugBrw4h6vd5FFzz9y7/2oULF/Loo4+yb98+Fi1axEMPPURlZSVr1qwhLS2NsWPHRh0yu6VoTxE7duzgxz/+MatWrWLgwIFcffXVHZ7HuagzDAC9M0R3/39SmH8bTYHMVpuaApkw/7Z2DhCRhDX/NkjLar0tLavb3+dFixbxyCOP8Oijj7Jw4UKqq6sZOnQoaWlpvPDCC+zateuUx5933nk89NBDAKxbt47S0lIAjhw5Qk5ODvn5+ezfv7/V4HrtDdl93nnn8cQTT3D8+HFqamr4y1/+woc//OFufb7O6PdPCk8E5/Jq47XczCOMtIOUu8HcHVrEh4Jzudzv4ESkc5rrAVfe4RUB5xd7CaGb9YNTp07l6NGjFBUVMWLECD7/+c9z6aWXUlJSwsyZMznjjFPP537DDTdwzTXXMGPGDGbOnMnZZ58NwJlnnsmsWbOYOnUq48ePZ+7cuZFjrr/+ei6++GJGjBjBCy+cmJl49uzZXH311ZFzXHvttcyaNavXZnOzUz2qJKKSkhLXUfvflk41U9Nriz/ak6GJSBds3LiRyZM1ykBPivY3NbM1zrmSjo7t98VH5VESwqm2i4gks36fFEYWZHVqu4hIMuv3SSEec7qKiPRX/b6iOR5zuopIz3LOtdsxTDqnu/XE/T4pQHzmdBWRnpGZmcnBgwcZPHiwEkM3Oec4ePAgmZmZHe/cjqRICiKSuIqLiykrK6OystLvUPqFzMxMiou73jlXSUFEfJWWlsa4ceP8DkPC+n1Fs4iIxE5JQUREIpQUREQkos8Nc2FmlcCpR6dqXyFwoAfDiYdEjzHR4wPF2BMSPT5I/BgTLb4xzrkhHe3U55JCd5jZ6ljG/vBToseY6PGBYuwJiR4fJH6MiR5fe1R8JCIiEUoKIiISkWxJ4T6/A4hBoseY6PGBYuwJiR4fJH6MiR5fVElVpyAiIqeWbE8KIiJyCkmTFMzsIjPbZGZbzWyx3/G0ZGajzOwFM9toZuvN7Ot+x9QeMwuY2dtm9je/Y4nGzArM7FEzey/89zzX75haMrNvhP+N15nZw2bW9ZHLei6mX5tZhZmta7FtkJk9a2ZbwsuBCRjjXeF/51Iz+4uZFSRSfC3e+5aZOTMr9CO2zkqKpGBmAeDnwMXAFOBzZjbF36haaQK+6ZybDJwDfDXB4mvp68BGv4M4hf8EnnHOnQGcSQLFamZFwE1AiXNuGhAAFvkbFQC/AS5qs20xsNI5NwFYGV730284OcZngWnOuRnAZuDW3g6qhd9wcnyY2SjgY8Du3g6oq5IiKQBnA1udc9udcw3AI8BlPscU4Zx73zn3Vvj1UbwLWcKN9W1mxcAlwAN+xxKNmeUB5wG/AnDONTjnqvyN6iSpQJaZpQLZQLnP8eCcexk41GbzZcBvw69/C1zeq0G1ES1G59w/nHNN4dU3gK4PDdpN7fwNAX4KfBvoM5W3yZIUioA9LdbLSMCLLoCZjQVmAf/0N5Ko7sb7Dx7yO5B2jAcqgQfDRVwPmFmO30E1c87tBX6Md9f4PlDtnPuHv1G1a5hz7n3wblqAoT7H05H/ATztdxAtmdkCYK9z7l2/Y+mMZEkK0WbuSLjMbWa5wGPAzc65I37H05KZfRKocM6t8TuWU0gFZgP3OudmATX4X+wRES6XvwwYB4wEcszsKn+j6vvM7Ht4RbAP+R1LMzPLBr4H3OZ3LJ2VLEmhDBjVYr2YBHhsb8nM0vASwkPOucf9jieKucACM9uJV/z2UTP7g78hnaQMKHPONT9lPYqXJBLFBcAO51ylc64ReBz4oM8xtWe/mY0ACC8rfI4nKjP7EvBJ4PMusdrXn4aX/N8Nf2eKgbfMbLivUcUgWZLCKmCCmY0zs3S8yr2nfI4pwrw5CH8FbHTO/cTveKJxzt3qnCt2zo3F+/s975xLqLtc59w+YI+ZTQpvmg9s8DGktnYD55hZdvjffD4JVBHexlPAl8KvvwQ86WMsUZnZRcB3gAXOueN+x9OSc26tc26oc25s+DtTBswO/x9NaEmRFMKVUTcCK/C+hMucc+v9jaqVucAX8O6+3wn/fMLvoPqorwEPmVkpMBP4oc/xRISfYB4F3gLW4n3/fO/1amYPA68Dk8yszMz+FVgKfMzMtuC1nlmagDH+P2AA8Gz4O/OLBIuvT1KPZhERiUiKJwUREYmNkoKIiEQoKYiISISSgoiIRCgpiIhIhJKCiIhEKCmIiEiEkoJIFGZ2lZm9Ge4U9cvwPBLHzOz/mtlbZrbSzIaE951pZm+0GNd/YHj76Wb2nJm9Gz7mNDMbYWYvh8+7zsw+7O8nFWlNSUGkDTObDHwWmOucmwkEgc8DOcBbzrnZwEvAkvAhvwO+Ex7Xf22L7Q8BP3fOnYk3xtH7wJXAivB5zwTe6Z1PJRKbVL8DEElA84GzgFXeEEVk4Q0IFwL+FN7nD8DjZpYPFDjnXgpv/y3wZzMbABQ55/4C4JyrAzCzVcCvwwMgPuGcU1KQhKInBZGTGfBb59zM8M8k59ztUfY71Rgx0YZrb56M5TxgL/B7M/tit6MV6UFKCiInWwksNLOhEJmveAze92VheJ8rgVedc9XA4RZ1A18AXgrPh1FmZpeHz5ERHh11DN68FPfjjYybSEN7i2hAPJFozOyzeHP+pgCNwFeB5/CmV/wEUA181jlXaWYzgV/gTa+5HbjGOXfYzCYAvwQKw+f4DPBh4Jbw+jHgi865Hb352URORUlBJEZmdsw5l+t3HCLxpOIjERGJ0JOCiIhE6ElBREQilBRERCRCSUFERCKUFEREJEJJQUREIpQUREQk4v8DlI9V8JSb9AQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model hyperparameters: ([100], 0.99, 15, 64, 0.0001, 0.01)\n",
      "best model average train and validation accuracy: 0.850125 0.850125\n"
     ]
    }
   ],
   "source": [
    "plt.plot(best_model.train_acc_history, '-o', label=\"train\")\n",
    "plt.plot(best_model.val_acc_history, '-o', label=\"validation\")\n",
    "plt.xlabel(\"epocs\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "print(\"best model hyperparameters: \"+str(best_combo));\n",
    "print(\"best model average train and validation accuracy:\",\\\n",
    "      np.mean(best_model.train_acc_history),np.mean(best_model.train_acc_history))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finished training model # 486/486"
     ]
    }
   ],
   "source": [
    "import itertools\n",
    "\n",
    "data = {\n",
    "      'X_train': feat_train,\n",
    "      'y_train': label_train,\n",
    "      'X_val': feat_val,\n",
    "      'y_val': label_val}\n",
    "\n",
    "best_val = 0.0\n",
    "best_model = None\n",
    "best_combo = None\n",
    "\n",
    "def train_model(combo):\n",
    "    hd,lrd,ne,bs,lre,ws = combo\n",
    "    model = FullyConnectedNet(input_dim=75,\n",
    "                              hidden_dims=hd,\n",
    "                              weight_scale=ws)\n",
    "    solver = Solver(model, data,\n",
    "                    update_rule='sgd',\n",
    "                    optim_config={\n",
    "                      'learning_rate': lre,\n",
    "                    },\n",
    "                    lr_decay=lrd,\n",
    "                    num_epochs=ne, \n",
    "                    batch_size=bs,\n",
    "                    verbose=False\n",
    "                    #print_every=100\n",
    "                   )\n",
    "    solver.train()\n",
    "    return solver\n",
    "    \n",
    "lr_decay = [0.95,0.99,1.0]\n",
    "num_epochs = [10,15]\n",
    "batch_size = [64,128,256]\n",
    "learning_rate = [1e-4,1e-3,1e-2]\n",
    "weight_scale = [1e-4,1e-3,1e-2]\n",
    "hidden_dims = [[h] for h in [75,100,125]]\n",
    "\n",
    "prog = 1\n",
    "combos = [combo for combo in itertools.product\\\n",
    "(hidden_dims,lr_decay,num_epochs,batch_size,learning_rate,weight_scale)]\n",
    "total = str(len(combos))\n",
    "\n",
    "for combo in combos:\n",
    "    model = train_model(combo)\n",
    "    avg_val = np.mean(model.val_acc_history)\n",
    "    if avg_val > best_val:\n",
    "        best_val = avg_val\n",
    "        best_model = model\n",
    "        best_combo = combo\n",
    "    print(\"\\rfinished training model # \"+str(prog)+\"/\"+total,end=\"\")\n",
    "    prog += 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYUAAAEKCAYAAAD9xUlFAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAIABJREFUeJzt3Xl8XHW9+P/XO5NlsidN0i1pSYFSaEtpS1i0ikBFikgB4QsV8QJXrF8WQe8VgXv9AXK/XnvFi4gisoiiIrXsRQuoUASUpa1ANygUKJ0kXdLMZGlmsk3evz/OZDpJJskkzWQmmffz8chj5pw5c+Y9afN5n8/nfBZRVYwxxhiAtEQHYIwxJnlYUjDGGBNmScEYY0yYJQVjjDFhlhSMMcaEWVIwxhgTZknBGGNMmCUFY4wxYZYUjDHGhKUnOoChKi0t1crKykSHYYwxY8qGDRv2qWrZYMeNuaRQWVnJ+vXrEx2GMcaMKSLycSzHWfORMcaYMEsKxhhjwiwpGGOMCbOkYIwxJsySgjHGmLAx1/vIGGOS2ZNv1nDbc9uobQgwtSib606fxTkLyhMdVswsKRhjzAh58s0abnx8E4GOIAA1DQFufHwTwJhJDJYUjDEp7WCu7DuCXTQGOmjwt9Pg7+DWP24NJ4RugY4gK555l7PnT0VE4vEVRpSMtTWaq6qq1AavGTM2xKMpZSTP2fvKHiArPY1LPnEIR04poMEfKvADHfj8Bwp/n7+dRn8HzW2dMX+WOyONacU5TJ+Qw7QJzuP0CTlML8lhWnEO2ZmuuH5nEdmgqlWDHWc1BWP6MRbahpM5xuhNKRuB4TelxNo8o6o0BTqpb2mjvqWd+v3dj+14W9rZt7+N+v3trNvhpbOr54VxW2cX9778UXhbBAqzMyjKzqAoJ5PSvEwOn5hHUU4GRdmZFOdmUJidQXFOJv/+yNvUNbf1ibswO4MLqirY6fWz0xvg9Y+87O+VUMrys8KJIjJpbN3VyP888y6Bjq4Bv/NIsZqCMVFEu4LMznDxgy8enbSFLhx8jENNMl1dSn1LO3uaWtnV2MruxkDosZU/btpFe2dXn/cIUJybiTs9DXemC3e6C3dGGtnh5y6yMtLIznCeuyOe//SF7TQGOvqcMzvDRVVlMfv2t+NtacPb0k5HMHrZlu9OpzQvi5LcTNZ/7It6jAAvfPtkinMyyHdn4EqLrdkn1n8TVcXn7wglCT8er5+d9f7w9q7GAF2DFM3lRdn8/YZTY4oLrKZgUtBQCrRAe5C9za3sbW6jrrmNvU0Rz5vb+Pv2fX2uIAMdQf6/pzZTmJ3BnKkFTCxwj8bXiqq1I8j317wTtf36pqc20+BvDxWoTqF64LkrVMCG9qW7cGemkelK46m3avtchd/w2EZ21LdwxKT8cKG/u6ktXPjvaWrtU/impwmTCtxREwKAAmfMnUxrRxetnUHaOoIEOoK0dnTR4O+gNfTceXReG6yADHQEaW7tpLzIzdHlBZSECv2SvExKcrPCj8W5GWSlH2imWbTiBWoaAn3ON7UomxmluTH8S/TU/f9tsP+HIsKE3Ewm5GYyf1pRn/O0d3axqzHATq+fr/zyjaifVRsl7pFgNQUzLkS7QstwCUvmTKYs383e5lbqIgr93lV3AFeaUJaXRVl+FptqGgf9zLL8LOZMLWDO1ALmTi1kztRCpk3IHtGbiW2dQT6sa+G9Pc2hn/28t6eZnV4/I/mnKwKoU2APJCs9jSmFbiYXuplc4GZyYXaP7SmFbkrysnClSb8F7lCvcFWVjqDS2hnkc7e/xO6m1oM+Z7exUCNctOIFjm36C99JX8VU2UetlvLDzgvYUHCa1RSMicbj9XPz6i19rpo7gsrTG3eRk+liYr5T2B81pYCTjnCeT8zPYmKBO/zahJxM0kLNBP0VaFMK3dxx4Xy21DaxubaRrbVNvPz+PoKhS9kCdzqzu5NEufN4aFleuPmhv9pMR7CLj/a1HCj4dzfz3t5mPq73h8/tShNmlOYyZ2oB5y4o5zf/2IHX37cpZUqhmz9d8+keV9qtHV20dQRp7QwSaA9dhXf2vCL/6Qvb+/0dP3Ptp5lc4KYoJyPmpHfd6bOiFrjXnT4rpvd3ExEy04XM9DRuOONIXnni53yTleEC8g6W8anTrxzSObvFemWfSHfMfp+5G+4nW9oBqJB9/E/G/WyeXQkMPREOxmoKZsxp7+xi3Q4va9/dy9pte/mgrqXfYwX4aMWZQ/6MoVxBtnYE2ba7mS21TWypbWRzbRPv7mqiLdR84s5I46gpBeRkpPHGDl+P5haXODWOffvbw81VaQKVJbnMnJTHEZPymTkpn1mT8plRmktm+oFJCEb6KnekruwjrVt9D9P+eRsTtY69UoZn4XUct/TrwzoXABtX0fnUN0gPHqgtdLrcpJ/9U5h3wfDPO5I2roLnb4XGaiisgMU3xR5bux/8+6BlH/i9zvNnrofWhr7HFk6Db22OOSyrKZhxZXdjKy9uc5LAK+/vo6U9SKYrjRMOncCXTziEe176gD1NfXt9TC3KHtbnDeUK0p3h4phpRRwT0TbcGezig7oWttQ2OrWKmkb+8YG3T/NMUKHB38Hykw4NJYA8DivLw53Rf/fE4cQYi5G6sg/buIrjNt0MBEBgMnVM3ngzTBI4fDF0+KEj4Dy2+yO2A9DR0uu10PP3nu2REABn+5nrnUKyaBrkT4G0wX9/kXEOuxCPdq6nr3HiBWj0wOpvwL73YMoxocK+3vkJPw8lgJZ90DmE+wSN1cOLcRBWUzBJqTPYxVueBl54dy9rt9Xxzq4mAKYWujn5yImcOmsinzishNws57omLm3DI1lYADNu+FPUNvvh1mbiYVhX9l1B2L8XmmqhqTr0WAPr7j9QOA5XRg5kZENGrvO4b9vg70lLh4KpUDjdSRLdyaKwwtlXWAEZoU4CvQtxcD7nrDv7/lt3toUK8lBh3lJ/oFDvLuDfew6CfS9O+sjMg5wJkFMKuaWQU3LgJ7wdeu3Bs5zfZ29WUzDjTe/29StOPpSczHTWbqvjpffqaAx04EoTqg4p5oYzjuSUWRM5YlJe1DbtEW8bjnbF9/Q1zvNhJoapRdn93jBMCtGu7DfdDJOAqQudgqm7wA8/r4XmXdDV68Z9uhs6+94QDjv3XqfwzcyJKPhzIn6ynZ/e/9Y/nuv8W/SWNxnOvgsadzpJvMHjHPfRy9BcC9qrJ1TuRCdR7Nna9+q8IwBPXwubHul5Rd++P/p3kTTInuAU5P0mBIGv/80p6HNKDiSlWHz2luiJa/FNsZ9jCOJaUxCRJcBPABdwv6qu6PX6IcADQBngBS5W1QHrRFZTGB+e2FDNjU9sojVKt8XSvCxOnlXGqUdOZNHhpRRmZ4xucMEOuONop7DrLbsEvngvuAvAXQhZocdoBVgv61bfw9wN3w3fMAQIaCabj/1/w29nH25tpr3FubrfvxdaQo9/vQXamgZ/b3q2cyVeMNX5zO7nBd3Py52r4DuOjl6AD/EKt4ehXNl3C3Y4yavRcyBZdD//cG3/nzXlmAOFeOTVfO8reXfhgeaq/pLWwXzn7u99kLXWWGsKcUsKIuIC3gNOA6qBdcCXVHVrxDGPAH9U1QdF5FTgMlX9ykDntaSQGEMd1NTdz7rGF6C6wXmsbQhQE/r5uN4f9X1l+Vm8fuPicC+gIRnKH05nG/g+Bu+HfX8adoIGo7+vP2npEUkiMmEUHdh+7e7oNwxzJ8KXVoIrHdIywBX66X6elt5r23Xg+/YuINPdsOhamDQ3VNjXHSj0wwmgzmmzH4qLVh0o8LOLB02A/cY3WAEei5Fs1hvpQjxe33kEJENS+ARwi6qeHtq+EUBVfxBxzBbgdFWtFqdNoFFVCwY6ryWF0Retvd6dkca1i2dy5OSCcKFf0xCgxuenpiHA3ua2Hv3oRWBifhblRdmUF+fw9Nu1UT9r2O3rUQvIbDj5eig9omehX/9hqCCICDCrACYceuBn/QMQ8Pb9nLzJcOFvobXJKeDbmqC1MbTd2P92f00PwyFpToIItjP4yAJCV7cTIa8s9Bj66X6eW+Y8/vK06DcvD/bKfgTvy4y4eBTiSfqdk+GeQjkQmYKrgRN6HfM2cB5OE9O5QL6IlKhqfeRBIrIcWA4wffr0uAU8ngx3TpzWjmBogFdreKDXbc9u6zMGoLWji/959sBNv0xXGlOK3JQXZXPSzDLKi7NDCSCbiqIcJhe6e3Sn/OfHvqG3r3cFnQI24Ov788L/63tTszPgNIl0y57gFPjTT4QJF/VMAjkTel79ls2KXlh87r9g2vGD/h6jxn7H0dFvGOaWOe3hwQ7o6oBgp1Pgd3WE9nX2fK2rw3n97z/p58NC7de5E53mDVeMzW+Lbx75tut5FyRFgdiv7thGshBP9u88iHgmhWj1y96XNd8GfiYilwIvATVAn6GmqnovcC84NYWRDXP8iTZp2A2Pb2R3Y4CjK4rChf7epraIqR2caR6aW6PP+rg07ZU+Bfjqrk/x2BWfpKI4m7K8rCE1+dx55CbmvHkfbnEGX1XIPv434x7qirbBn/4UveBvHXyUcVRfWwsTZjjNHrEa6cIizdX/DcPT/xuOOH3o59z8eD9NHxVOe/hQxaOAHAvGeCE+0hLafNTr+DzgXVWtGOi81nw0uP4GIfWWneFiYkEWZXlZTCzIYmK+m7L8iNG+oe2f/eS/ub7j5+RE3CD1ayY/zLiSW777PWeHKrQ1Q0tdz5uXLXWwf0/Ptu2WOqfPeb+BTXAK8KH83HvSyDd9xEM8+8RD0rRfm+STDM1H64CZIjIDpwawDLgo8gARKQW8qtoF3IjTE8kcpIEmylq5/MRwoZ+XlR7TlAXXZ/yBnM72HvtypJ3vdt0N971y4OZl1IE34rRpd7ddTzveadZ47a5+Pk3g+o/6eW0A8Wj6iIeRvCpN1St7E1dxSwqq2ikiVwPP4XRJfUBVt4jIrcB6VV0NnAz8QEQUp/noqnjFkyr2NreS7pKo0waXF2Vz4qElsZ2osRo8b0D1enICUbpmAuldbZCV57TJ97h5GXFDM6fU6VXT2zur+2/6GI5ULSCt6cOMMBvRPI5srW3i8gfXUdfchgi0RySGAUf3drTCrreh+o1wIqA51Dso3e0M/Am2933faPc3N8YMWzI0H5lR9Pw7e7jm4TfJd2fwxFWLCGxY2Xe6ggVLnLb/Rs+Bwr/6Ddi10enRAlA0HQ75pNPMU1EFk46GrU/Gp1cKpN6VvTFJzmoKY5yq8stXPuL7a95h7tRC7r+kikk7VvctxNMyYNIcaN4N+3c7+9KzoXwhVBx34Cd/UvQPStK+18aY2FhNIQV0BLu46aktPPzGTpbMmcyPL5zvLP79/K19++x3dcCeTTDnvFAt4DgnScTah93aro1JCZYUxqhGfwdX/n4Df99ez5UnH8a3PzfrwDiB/qbU7eqC8+4bvSCNMWOOJYUxaMe+Fv71wXV4vH5+9H+O4fxjI3rsbF3d/xuH27PHGJMyLCmMMa9/WM/Xf7cBgN999QRO6O5iGuxwpnR49WdQVOncN4icujgZ++wbY5JO2uCHmGTxyHoPF//ydSbkZvLklYsOJITGGvj1mU5COP7rcPU6WPpTp8so4jxaV09jTAyspjAGdHUpt/15G3e/+AGfOryUuy5aSGFO6AbxBy/AY5c7U0Gf/wDMPc/ZbzeGjTHDYEkhyfnbO/m3P7zNs1t2c9EJ0/ne0jlkuNKcWTdfug1eXAFlR8IFv4GyIxIdrjFmjLOkkMT2NLVy+YPr2VLbyE1fmM1liyqduYpa6uHxy51awrxl8IXbITM30eEaY8YBSwpJanNNI5c/uJ7m1g7uv6SKU48MDSrzvAGPXOqsG3vWT2DhJbGtgmWMMTGwpJAkIhfFKc7NpDnQwcQCN49e8UmOmlLgTE/x+i/gz991lkT86p9h6vxEh22MGWcsKQzTcFc26+9ckYvieFvaEYGvf2aGkxBam+Cpq5yZRWedCef8HLKLRvLrGGMMYElhWKKtbHbj45sA+iSGzmAXza2dNLV2OI+BDppaO2luPfB438sf9lnuUhXu+dtH/MuMFlj1L+DbAaf9F3zyG9ZcZIyJG0sKw3Dbc33XLA50BLn+sY385tUdBwr9QGef44bik83PwP2/AXcRXPpHZ/ZSY4yJo7gmBRFZAvwEZ5Gd+1V1Ra/XpwMPAkWhY25Q1TXxjGkk9LeyWVtnFzmZ6UwqcFPgziDfnU5BdugxynaBO4M8dzon/XBtj0Xsd2kJO3Uin3C9A9NOgvN+6SxYY4wxcRa3pCAiLuAu4DSgGlgnIqtVdWvEYd8FVqnq3SIyG1gDVMYrppEytSg76hrI5UXZ/O7yE4Z8vjtmv8/cDfeTHVoDuVzqKaeemsmLKf/KI86i78YYMwriOc3F8cB2Vf1QVduBlcDZvY5RoCD0vBCojWM8I+a602eRntazXT87w8V1p88a1vmO++Cn4YQQqTzwniUEY8yoimdSKAciF+GtDu2LdAtwsYhU49QSvhHHeEbMOQvKOW5GMSIgODWEfpe6HEy7P/paxdD/FNjGGBMn8bynEK2LTO9l3r4E/FpV/1dEPgH8VkTmqmpXjxOJLAeWA0yfPj0uwQ5Veloac6cW8vQ3PjW8E+zfC2/cB+vu7/8Ym+raGDPK4pkUqoFpEdsV9G0e+iqwBEBVXxURN1AK7I08SFXvBe4FZznOeAU8FNW+AEdNyR/6G+vec2YzfXslBNvhyDOdFdD+cefIroFsjDHDEM+ksA6YKSIzgBpgGXBRr2N2AouBX4vIUYAbqItjTCOiq0up8QX43Ox+1jPuTRU+/jv846fw3rOQ7oYFX4YTr4LSw51jSg63NZCNMQkXt6Sgqp0icjXwHE530wdUdYuI3AqsV9XVwL8D94nIt3Cali5V1aSoCQxkT3Mr7cEuKibkDHxgsBPeecpJBrVvQk4pnHwjHHc55Jb2PNamujbGJIG4jlMIjTlY02vfTRHPtwKL4hlDPHi8TjPPtOLs6Ae0NcM/fwuv3Q2NO51awBfugGOWOc1CxhiTpGxE8zB4vH4ApvWuKTTVwuv3wPpfQVsjTP8knPE/cMQSSLNF7owxyc+SwjB4fH6Wpr3CjN9+B5pqnNHGxYdCzXrQIBy11JmjqKIq0aEaY8yQWFIYhtIPV/P1zF+S1tTm7Ni/x/k5bLGz4E1xZULjM8aY4bI2jWE4ffc9ZNPW94V971lCMMaMaZYUhqGkq59eszYC2RgzxllSGKL2zi5qtST6izYC2RgzxllSGKLahgA/7LiALuk1UZ2NQDbGjAOWFIbI4/OzuutT+AtnQloGIFA4Dc660wafGWPGPOt9NETdA9eypBNmnQEX/jbBERljzMixmsIQeXx+Ml1KetNO62lkjBl3LCkMkcfrZ16BHwm2W1Iwxow7lhSGyOMLMD+v0dmYMCOxwRhjzAizpDBEHq+fI7P2ORtWUzDGjDOWFIagpa0Tb0s7h6TVgbicXkfGGDOOWFIYAo/PmR11StduZ6CaKyPBERljzMiypDAE3d1Ri9tqrOnIGDMuxTUpiMgSEdkmIttF5IYor/9YRN4K/bwnIg3xjOdgda+jkL2/2pKCMWZcitvgNRFxAXcBpwHVwDoRWR1abQ0AVf1WxPHfABbEK56R4PH5KcloJy2wz5KCMWZcimdN4Xhgu6p+qKrtwErg7AGO/xLwcBzjOWgeb4DjCq07qjFm/IpnUigHPBHb1aF9fYjIIcAM4IV+Xl8uIutFZH1dXT/TVo+Cap+fudk+Z8NqCsaYcSieSUGi7NN+jl0GPKqqwWgvquq9qlqlqlVlZWUjFuBQqCoer58jMm2MgjFm/IpnUqgGIjvyVwC1/Ry7jCRvOvL5O2hpD1LBHnAXQXZxokMyxpgRF8+ksA6YKSIzRCQTp+Bf3fsgEZkFFAOvxjGWg9bd82hi5y6rJRhjxq24JQVV7QSuBp4D3gFWqeoWEblVRJZGHPolYKWq9te0lBS6B67lB6w7qjFm/IrregqqugZY02vfTb22b4lnDCPF4w2QRheZ+6thwrmJDscYY+LCRjTHyOPzMyu7CenqtJqCMWbcsqQQI4/Xz8L80IBrSwrGmHHKkkKMqn0BZru9zkaxDVwzxoxPlhRi0NWl1PgCzEivg7R0KIg6Bs8YY8Y8Swox2NPcSnuwi6lde5w1FFxxvT9vjDEJY0khBt1TZk9or7E5j4wx45olhRh0D1zLbfHYTWZjzLhmSSEGHp+fAmnB1dZgScEYM65ZUoiBxxtgQa51RzXGjH+WFGLg8fmZl9c9ZbbdUzDGjF+WFGLgTJld72wUH5LYYIwxJo5iSgoi8piInCkiKZdE2jqD7G5q5RDZC9kTwF2Y6JCMMSZuYi3k7wYuAt4XkRUicmQcY0oqtQ2tqMKk4C7rjmqMGfdiSgqq+ldV/TKwENgB/EVE/iEil4lIRjwDTLTu7qiFrbV2k9kYM+7F3BwkIiXApcDlwJvAT3CSxF/iElmS8Pj8pNNJVkuNJQVjzLgX6z2Fx4GXgRzgLFVdqqp/UNVvAHkDvG+JiGwTke0ickM/x1wgIltFZIuI/H44XyKePN4A011eRIPW88gYM+7FOonPz1T1hWgvqGpVtP0i4gLuAk7DWa95nYisVtWtEcfMBG4EFqmqT0QmDin6UeDx+Zmf3witWE3BGDPuxdp8dJSIFHVviEixiFw5yHuOB7ar6oeq2g6sBM7udczXgLtU1QegqntjjGfUVHv9zAlPmV2Z0FiMMSbeYk0KX1PVhu6NUCH+tUHeUw54IrarQ/siHQEcISJ/F5HXRGRJjPGMGo8vwOEZdeDKhIKpiQ7HGGPiKtbmozQREVVVCDcNZQ7yHomyT6N8/kzgZKACeFlE5kYmoNDnLQeWA0yfPj3GkA9eS1sn3pZ2yov3QNF0SHON2mcbY0wixFpTeA5YJSKLReRU4GHg2UHeUw1Mi9iuAGqjHPOUqnao6kfANpwk0YOq3quqVapaVVZWFmPIB8/jc7qjlnbusqYjY0xKiDUpXA+8AFwBXAU8D3xnkPesA2aKyAwRyQSWAat7HfMkcAqAiJTiNCd9GGNMceeso6Dk+T3W88gYkxJiaj5S1S6cUc13x3piVe0Ukatxahku4AFV3SIitwLrVXV16LXPichWIAhcp6r1Q/0S8eLx+imkhfT2ZqspGGNSQkxJIdR19AfAbMDdvV9VDx3ofaq6BljTa99NEc8V+LfQT9Lx+PwckbnP2bCkYIxJAbE2H/0Kp5bQidPc8xvgt/EKKlnYOgrGmFQTa1LIVtXnAVHVj1X1FuDU+IWVHKp9fma5raZgjEkdsXZJbQ1Nm/1+6D5BDZB0o49Hkqri8fqpLKuD3DLI6nc2D2OMGTdirSl8E2feo2uAY4GLgUviFVQy8Pk7aGkPMqVrt9USjDEpY9CaQmig2gWqeh2wH7gs7lElge4ps4vbaqFiUYKjMcaY0TFoTUFVg8CxIhJthPK45fH5yaATd8AGrhljUkes9xTeBJ4SkUeAlu6dqvp4XKJKAh5vgHKpQ7TLkoIxJmXEmhQmAPX07HGkwPhNCj4/s91e51vaMpzGmBQR64jmlLiPEMnj9fPJHJ9TL7KagjEmRcQ6ovlX9J3hFFX91xGPKEl4vH5mZtSDKwvyJic6HGOMGRWxNh/9MeK5GziXvjOejhvBLqWmIcC0iaHuqGkxL2VtjDFjWqzNR49FbovIw8Bf4xJREtjT1EpHUCnr3A2llYkOxxhjRs1wL4FnAqO32s0oc8YoKAWBarufYIxJKbHeU2im5z2F3ThrLIxLHl+ACTST3tliPY+MMSkl1uaj/HgHkkw8Xj+HpO11NqymYIxJITE1H4nIuSJSGLFdJCLnxC+sxPL4/MzN9jkblhSMMSkk1nsKN6tqY/eGqjYANw/2JhFZIiLbRGS7iNwQ5fVLRaRORN4K/Vwee+jxU+0NMDs7tABc0SGJDcYYY0ZRrF1SoyWPAd8bmkjvLuA0oBpYJyKrVXVrr0P/oKpXxxjHqPD4/BzqrnPGJ2TmJDocY4wZNbHWFNaLyO0icpiIHCoiPwY2DPKe44HtqvqhqrYDK4GzDybY0dDWGWR3UytTdY81HRljUk6sSeEbQDvwB2AVEACuGuQ95YAnYrs6tK+380Rko4g8KiLTop1IRJaLyHoRWV9XVxdjyMNT29CKKpS011pSMMaknFh7H7UAfe4JDCLaVNu9p8p4GnhYVdtE5P8CDxJlmU9VvRe4F6CqqqrPdBsjyeP1k0U72a17rDuqMSblxNr76C8iUhSxXSwizw3ytmog8sq/gl5TY6hqvaq2hTbvw1nVLaE8Pj/lsg9BraZgjEk5sTYflYZ6HAGgqj4GX6N5HTBTRGaISCawDFgdeYCITInYXAq8E2M8cePxBjjUZWMUjDGpKdbeR10iMl1VdwKISCVRZk2NpKqdInI18BzgAh5Q1S0iciuwXlVXA9eIyFKgE/AClw7rW4wgj8/P0Tk+5w5KsTUfGWNSS6xJ4T+BV0Tkb6Htk4Dlg71JVdcAa3rtuyni+Y3AjTHGMCqqvX6+kFkPXdmQN1hlyBhjxpeYmo9U9VmgCtiG0wPp33F6II07Hl+A6Wl1TtNRai1LbYwxMU+IdzlwLc7N4reAE4FXidJTaCxraevE29LOZHctTJiV6HCMMWbUxXqj+VrgOOBjVT0FWADEd8BAAnh8zpTZha02RsEYk5piTQqtqtoKICJZqvouMO4upT3eAKU0kR4MWFIwxqSkWG80V4fGKTwJ/EVEfIzD5Tg9Xj/TZY+zYUnBGJOCYh3RfG7o6S0ishYoBJ6NW1QJ4vH5mZmxz9mw7qjGmBQUa00hTFX/NvhRY5PH6+ekbC+0AUXjdrVRY4zp15CTwnjm8QY4PKMOMqdChjvR4RhjzKiL9UbzuKeqzrxHutcmwjPGpCxLCiHelnb87UFKO607qjEmdVlSCPH4AmTRTm5bnSUFY0zKsqQQ4vH6mSbds6Na85ExJjVZUgjx+PxMF5sy2xiT2iwphHi8AY7Mqnc2LCkYY1KUJYWQap/fSQqZeZBbmuhwjDEmIeKaFERkiYhsE5HtItLvGs8icr538x0VAAATt0lEQVSIqIhUxTOegXi8fma4bMpsY0xqi1tSEBEXcBdwBjAb+JKIzI5yXD5wDfB6vGIZTLBLqWkIMLlrtzUdGWNSWjxrCscD21X1Q1VtB1YCZ0c57r+AHwKtcYxlQHuaWukMBiluszEKxpjUFs+kUA54IrarQ/vCRGQBME1V/xjHOAbl8fopo5H0rjZLCsaYlBbPpBCtYV7DL4qkAT/GWdpz4BOJLBeR9SKyvq5u5Nf28fgCEVNm2xgFY0zqimdSqAamRWxX0HMNhnxgLvCiiOzAWeJzdbSbzap6r6pWqWpVWVnZiAfq8fo5JM3GKBhjTDyTwjpgpojMEJFMYBmwuvtFVW1U1VJVrVTVSuA1YKmqro9jTFF5fH5mu72A2JTZxpiUFrekoKqdwNXAc8A7wCpV3SIit4rI0nh97nBUewPO4jqFFZCemehwjDEmYeK6noKqrgHW9Np3Uz/HnhzPWAbi8fmZlrbHmo6MMSkv5Uc0t3UG2d3UysTOXZYUjDEpL+WTQm1DK25tJbfDa0nBGJPyUj4peLw2O6oxxnSzpBA5ZbYtw2mMSXEpnxR2ev1UukID4mzgmjEmxaV8Uqj2Bjgqqx6yCiC7ONHhGGNMQsW1S+pY4PH5OSy9DooqbcpsY0zKS/magsfrZ4raGAVjjIEUTwr72zpp8LdR0m5jFIwxBlI8KXi8fibjw6Ud1vPIGGOwpGBjFIwxJkJqJwVfgOlp3esoVCY0FmOMSQapnRS8fg5Pr0PFBYXTBn+DMcaMcymdFKp9fo7IrEcKK8CVkehwjDEm4VI6KXi8Aaan1VnTkTHGhKRsUlBVPD4/k4PWHdUYY7rFNSmIyBIR2SYi20Xkhiiv/18R2SQib4nIKyIyO57xRPK2tJPW3kxuZ4N1RzXGmJC4JQURcQF3AWcAs4EvRSn0f6+qR6vqfOCHwO3xiqc3jy/ANOmeCK9ytD7WGGOSWjxrCscD21X1Q1VtB1YCZ0ceoKpNEZu5gMYxnh6cMQrWHdUYYyLFc0K8csATsV0NnND7IBG5Cvg3IBM4NdqJRGQ5sBxg+vTpIxJcj3UUbMpsY4wB4ltTiDblaJ+agKrepaqHAdcD3412IlW9V1WrVLWqrKxsRILzeAPMzKwHdxFkF43IOY0xZqyLZ1KoBiJHhFUAtQMcvxI4J47x9FDtcwauWdORMcYcEM+ksA6YKSIzRCQTWAasjjxARGZGbJ4JvB/HeHrweP1UsMd6HhljTIS43VNQ1U4RuRp4DnABD6jqFhG5FVivqquBq0Xks0AH4AMuiVc8kYJdyq6GFkoyd1tNwRhjIsR15TVVXQOs6bXvpojn18bz8/uzp6mV0uA+XBq0pGCMMRFSckSzx+tnepr1PDLGmN5SMinstHUUjDEmqpRMCh5fgOmyB01Lh4LyRIdjjDFJIyWTQrXXz6zMfUjhNHDF9baKMcaMKSlZInp8firT6qw7qjFJoKOjg+rqalpbWxMdyrjgdrupqKggI2N4a8SkZlLwBpiie6D404kOxZiUV11dTX5+PpWVlYhEmwjBxEpVqa+vp7q6mhkzhnfRm3LNR22dQfzN9eQGm+wmszFJoLW1lZKSEksII0BEKCkpOahaV8olhRpfgGlYd1RjkoklhJFzsL/LlEsKTs8j645qjHE0NDTw85//fMjv+/znP09DQ0McIkqs1EsKNkbBmDHtyTdrWLTiBWbc8CcWrXiBJ9+sOajz9ZcUgsHggO9bs2YNRUXjb4bllLvR7PH5mZG2F80pQdwFiQ7HGDMET75Zw42PbyLQ4RTYNQ0Bbnx8EwDnLBjemKMbbriBDz74gPnz55ORkUFeXh5TpkzhrbfeYuvWrZxzzjl4PB5aW1u59tprWb58OQCVlZWsX7+e/fv3c8YZZ/CpT32Kf/zjH5SXl/PUU0+RnZ09Ml96lKVcUqj2Bjgtcx9itQRjks73nt7C1tqmfl9/c2cD7cGuHvsCHUG+8+hGHn5jZ9T3zJ5awM1nzen3nCtWrGDz5s289dZbvPjii5x55pls3rw53HvngQceYMKECQQCAY477jjOO+88SkpKepzj/fff5+GHH+a+++7jggsu4LHHHuPiiy+O9WsnldRrPvL5nRvNlhSMGXN6J4TB9g/H8ccf36M755133skxxxzDiSeeiMfj4f33+87wP2PGDObPnw/Asccey44dO0YsntGWcjWFXfVNlOpe63lkTBIa6IoeYNGKF6hpCPTZX16UzR++/okRiSE3Nzf8/MUXX+Svf/0rr776Kjk5OZx88slRu3tmZWWFn7tcLgKBvjGOFSlVU9jf1om7dTcubMpsY8ai606fRXaGq8e+7AwX150+a9jnzM/Pp7m5OeprjY2NFBcXk5OTw7vvvstrr7027M8ZK+JaUxCRJcBPcBbZuV9VV/R6/d+Ay4FOoA74V1X9OF7xWM8jY8a27pvJtz23jdqGAFOLsrnu9FnDvskMUFJSwqJFi5g7dy7Z2dlMmjQp/NqSJUv4xS9+wbx585g1axYnnnjiQX+HZBe3pCAiLuAu4DSc9ZrXichqVd0acdibQJWq+kXkCuCHwIXxisnj9XNId1KweY+MGZPOWVB+UEkgmt///vdR92dlZfHMM89Efa37vkFpaSmbN28O7//2t789orGNtng2Hx0PbFfVD1W1HVgJnB15gKquVVV/aPM1oCKO8YQHrqkrE/KnxPOjjDFmTIpnUigHPBHb1aF9/fkqED0ljxCP188M114omg5prsHfYIwxKSae9xSiTcChUQ8UuRioAj7Tz+vLgeUA06dPH3ZA1T4/h6XXIcWHDvscxhgznsWzplANTIvYrgBqex8kIp8F/hNYqqpt0U6kqveqapWqVpWVlQ07IE+9n6lde6w7qjHG9COeSWEdMFNEZohIJrAMWB15gIgsAO7BSQh74xgLqkpTQx052mI9j4wxph9xSwqq2glcDTwHvAOsUtUtInKriCwNHXYbkAc8IiJvicjqfk53UJ58s4ZPrHiBsg6novJ6Q348PsYYY8a8uA5eU9U1qnqEqh6mqt8P7btJVVeHnn9WVSep6vzQz9KBzzh03RNo7W5sDY9R+P6rbQc9s6IxJjXl5eUBUFtby/nnnx/1mJNPPpn169cPeJ477rgDv98f3k6WqbjH/Yjm257bFp5RsTspbO8o4bbntiUyLGPMcG1cBT+eC7cUOY8bVyUkjKlTp/Loo48O+/29k0KyTMU97pNCbcQ8KdNkL3VagB93j/3GmDFi4yp4+hpo9ADqPD59zUElhuuvv77Hegq33HIL3/ve91i8eDELFy7k6KOP5qmnnurzvh07djB37lwAAoEAy5YtY968eVx44YU95j664oorqKqqYs6cOdx8882AM8lebW0tp5xyCqeccgrgTMW9b98+AG6//Xbmzp3L3LlzueOOO8Kfd9RRR/G1r32NOXPm8LnPfS4ucyyN+wnxphZlhyfQOkT2sFMnhfcbY5LMMzfA7k39v169DoK9Oil2BOCpq2HDg9HfM/loOGNF9NeAZcuW8c1vfpMrr7wSgFWrVvHss8/yrW99i4KCAvbt28eJJ57I0qVL+13q8u677yYnJ4eNGzeyceNGFi5cGH7t+9//PhMmTCAYDLJ48WI2btzINddcw+23387atWspLS3tca4NGzbwq1/9itdffx1V5YQTTuAzn/kMxcXFozJF97ivKUROoDU9bS87deJBT6BljEmQ3glhsP0xWLBgAXv37qW2tpa3336b4uJipkyZwn/8x38wb948PvvZz1JTU8OePXv6PcdLL70ULpznzZvHvHnzwq+tWrWKhQsXsmDBArZs2cLWrVv7Ow0Ar7zyCueeey65ubnk5eXxxS9+kZdffhkYnSm6x31NoXuOlB8/u5kprfU0ZE7lB184esTnTjHGjIABrugB5x5Co6fv/sJpcNmfhv2x559/Po8++ii7d+9m2bJlPPTQQ9TV1bFhwwYyMjKorKyMOmV2pGi1iI8++ogf/ehHrFu3juLiYi699NJBz6MadYwvMDpTdI/7mgLAOa6/87fMa3GJclnm85zj+nuiQzLGDMfimyCjV9NvRraz/yAsW7aMlStX8uijj3L++efT2NjIxIkTycjIYO3atXz88cCTN5900kk89NBDAGzevJmNGzcC0NTURG5uLoWFhezZs6fH5Hr9Tdl90kkn8eSTT+L3+2lpaeGJJ57g05/+9EF9v6EY9zWF8I2pjlBGDXidbYB5FyQuLmPM0HX/zT5/KzRWQ2GFkxAO8m95zpw5NDc3U15ezpQpU/jyl7/MWWedRVVVFfPnz+fII48c8P1XXHEFl112GfPmzWP+/Pkcf/zxABxzzDEsWLCAOXPmcOihh7Jo0aLwe5YvX84ZZ5zBlClTWLt2bXj/woULufTSS8PnuPzyy1mwYMGoreYmA1VVklFVVZUO1v+3h4Gqm9/a3He/MWZUvfPOOxx11FGJDmNcifY7FZENqlo12HvHf/NRY/XQ9htjTAob/0mhsJ8lGvrbb4wxKWz8J4U43ZgyxpjxaPwnhXkXwFl3OvcQEOfxrDvtJrMxSWSs3dtMZgf7uxz/vY/ASQCWBIxJSm63m/r6ekpKSvodMWxio6rU19fjdruHfY7USArGmKRVUVFBdXU1dXV1iQ5lXHC73VRUDP+eqSUFY0xCZWRkMGOGrYaYLMb/PQVjjDExs6RgjDEmzJKCMcaYsDE3zYWI1AEDz07Vv1Jg3wiGEw/JHmOyxwcW40hI9vgg+WNMtvgOUdWywQ4ac0nhYIjI+ljm/kikZI8x2eMDi3EkJHt8kPwxJnt8/bHmI2OMMWGWFIwxxoSlWlK4N9EBxCDZY0z2+MBiHAnJHh8kf4zJHl9UKXVPwRhjzMBSraZgjDFmACmTFERkiYhsE5HtInJDouOJJCLTRGStiLwjIltE5NpEx9QfEXGJyJsi8sdExxKNiBSJyKMi8m7o9/mJRMcUSUS+Ffo33iwiD4vI8GcuG7mYHhCRvSKyOWLfBBH5i4i8H3osTsIYbwv9O28UkSdEpCiZ4ot47dsioiJSmojYhiolkoKIuIC7gDOA2cCXRGR2YqPqoRP4d1U9CjgRuCrJ4ot0LfBOooMYwE+AZ1X1SOAYkihWESkHrgGqVHUu4AKWJTYqAH4NLOm17wbgeVWdCTwf2k6kX9M3xr8Ac1V1HvAecONoBxXh1/SNDxGZBpwG7BztgIYrJZICcDywXVU/VNV2YCVwdoJjClPVXar6z9DzZpyCrDyxUfUlIhXAmcD9iY4lGhEpAE4Cfgmgqu2q2pDYqPpIB7JFJB3IAWoTHA+q+hLg7bX7bODB0PMHgXNGNaheosWoqn9W1c7Q5mtAwpZT7Od3CPBj4DvAmLl5mypJoRzwRGxXk4SFLoCIVAILgNcTG0lUd+D8B+9KdCD9OBSoA34VauK6X0RyEx1UN1WtAX6Ec9W4C2hU1T8nNqp+TVLVXeBctAATExzPYP4VeCbRQUQSkaVAjaq+nehYhiJVkkK0lTuSLnOLSB7wGPBNVW1KdDyRROQLwF5V3ZDoWAaQDiwE7lbVBUALiW/2CAu1y58NzACmArkicnFioxr7ROQ/cZpgH0p0LN1EJAf4T2DMrfubKkmhGpgWsV1BElTbI4lIBk5CeEhVH090PFEsApaKyA6c5rdTReR3iQ2pj2qgWlW7a1mP4iSJZPFZ4CNVrVPVDuBx4JMJjqk/e0RkCkDocW+C44lKRC4BvgB8WZOrf/1hOMn/7dDfTAXwTxGZnNCoYpAqSWEdMFNEZohIJs7NvdUJjilMnDUIfwm8o6q3JzqeaFT1RlWtUNVKnN/fC6qaVFe5qrob8IjIrNCuxcDWBIbU207gRBHJCf2bLyaJboT3shq4JPT8EuCpBMYSlYgsAa4HlqqqP9HxRFLVTao6UVUrQ38z1cDC0P/RpJYSSSF0M+pq4DmcP8JVqrolsVH1sAj4Cs7V91uhn88nOqgx6hvAQyKyEZgP/HeC4wkL1WAeBf4JbML5+0v4qFcReRh4FZglItUi8lVgBXCaiLyP03tmRRLG+DMgH/hL6G/mF0kW35hkI5qNMcaEpURNwRhjTGwsKRhjjAmzpGCMMSbMkoIxxpgwSwrGGGPCLCkYY4wJs6RgjDEmzJKCMVGIyMUi8kZoUNQ9oXUk9ovI/4rIP0XkeREpCx07X0Rei5jXvzi0/3AR+auIvB16z2EiMkVEXgqdd7OIfDqx39SYniwpGNOLiBwFXAgsUtX5QBD4MpAL/FNVFwJ/A24OveU3wPWhef03Rex/CLhLVY/BmeNoF3AR8FzovMcAb43OtzImNumJDsCYJLQYOBZY50xRRDbOhHBdwB9Cx/wOeFxECoEiVf1baP+DwCMikg+Uq+oTAKraCiAi64AHQhMgPqmqlhRMUrGagjF9CfCgqs4P/cxS1VuiHDfQHDHRpmvvXozlJKAG+K2I/MtBR2vMCLKkYExfzwPni8hECK9XfAjO38v5oWMuAl5R1UbAF3Fv4CvA30LrYVSLyDmhc2SFZkc9BGddivtwZsZNpqm9jbEJ8YyJRkQuxFnzNw3oAK4C/oqzvOLngUbgQlWtE5H5wC9wltf8ELhMVX0iMhO4BygNneP/AJ8Grgtt7wf+RVU/Gs3vZsxALCkYEyMR2a+qeYmOw5h4suYjY4wxYVZTMMYYE2Y1BWOMMWGWFIwxxoRZUjDGGBNmScEYY0yYJQVjjDFhlhSMMcaE/f/yy/14HecJywAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "best model hyperparameters: ([125], 0.99, 15, 64, 0.0001, 0.01)\n",
      "best model average train and validation accuracy: 0.8689375 0.8689375\n"
     ]
    }
   ],
   "source": [
    "plt.plot(best_model.train_acc_history, '-o', label=\"train\")\n",
    "plt.plot(best_model.val_acc_history, '-o', label=\"validation\")\n",
    "plt.xlabel(\"epocs\")\n",
    "plt.ylabel(\"accuracy\")\n",
    "plt.legend()\n",
    "plt.show()\n",
    "print(\"best model hyperparameters: \"+str(best_combo));\n",
    "print(\"best model average train and validation accuracy:\",\\\n",
    "      np.mean(best_model.train_acc_history),np.mean(best_model.train_acc_history))"
   ]
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
